package com.kd.mining.lda.core;

import java.io.File;
import java.io.FileReader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.wltea.analyzer.core.KDAnalyzer;
import org.wltea.analyzer.word.Word;

import com.kd.utils.FileUtil;
import com.kd.utils.StringUtils;

public class Documents {
	
	// document collection
	private static List<Document> docs;
	// voc
	private static List<String> indexToTerm;
	// term to index
	private static Map<String, Integer> indexByTerm;
	// number of term
	private static Map<String, Integer> countByTerm;
	
	public Documents() {
		docs = new ArrayList<Document>();
		indexToTerm = new ArrayList<String>();
		indexByTerm = new HashMap<String, Integer>();
		countByTerm = new HashMap<String, Integer>();
	}
	
	public void load(String dir) {
		if(StringUtils.isBlank(dir)) {
			throw new RuntimeException("The original document directory to load is null!");
		}
		String path = Documents.class.getClassLoader().getResource(dir).getPath();
		File[] files = new File(path).listFiles();
		if(null == files) {
			return ;
		}
		for(File docFile : files){
			Document doc = new Document(docFile.getPath(), indexToTerm, indexByTerm, countByTerm);
			docs.add(doc);
		}
	}
	
	public List<Document> getDocs() {
		return docs;
	}

	public void setDocs(List<Document> docs) {
		Documents.docs = docs;
	}

	public List<String> getIndexToTerm() {
		return indexToTerm;
	}

	public void setIndexToTerm(List<String> indexToTerm) {
		Documents.indexToTerm = indexToTerm;
	}

	public Map<String, Integer> getIndexByTerm() {
		return indexByTerm;
	}

	public void setIndexByTerm(Map<String, Integer> indexByTerm) {
		Documents.indexByTerm = indexByTerm;
	}

	public Map<String, Integer> getCountByTerm() {
		return countByTerm;
	}

	public void setCountByTerm(Map<String, Integer> countByTerm) {
		Documents.countByTerm = countByTerm;
	}

	class Document {
		
		private String docName = null;
		int[] terms = null;
		
		public Document() {
			
		}
		public Document(String docName, List<String> indexToTerm, Map<String, Integer> indexByTerm, Map<String, Integer> countByTerm) {
			this.docName = docName;
			List<String> words = dealChinese(docName);
			//Transfer word to index
			this.terms = new int[words.size()];
			for(int i = 0; i < words.size(); i++){
				String word = words.get(i);
				if(!indexByTerm.containsKey(word)){
					int newIndex = indexByTerm.size();
					indexByTerm.put(word, newIndex);
					indexToTerm.add(word);
					countByTerm.put(word, new Integer(1));
					terms[i] = newIndex;
				} else {
					terms[i] = indexByTerm.get(word);
					countByTerm.put(word, countByTerm.get(word) + 1);
				}
			}
			words.clear();
		}
		
		public List<String> dealChinese(String docName) {
			List<String> terms = new ArrayList<String>();
			String text = FileUtil.fileToString(docName);
			KDAnalyzer analyzer = new KDAnalyzer(null, new StringReader(text), true);
			List<Word> words = analyzer.getWords();
			for(Word word : words) {
				if("CN_WORD".equals(word.type()) && !StringUtils.isBlank(word.text()) && word.text().length()>1) {
					terms.add(word.text());
				}
			}
			return terms;
		}
		
		public List<String> dealEnglish(String docName) {
			//Read file and initialize word index array
			ArrayList<String> docLines = new ArrayList<String>();
			ArrayList<String> words = new ArrayList<String>();
			FileUtil.readLines(docName, docLines);
			for(String line : docLines){
				FileUtil.tokenizeAndLowerCase(line, words);
			}
			//Remove stop words and noise words
			for(int i = 0; i < words.size(); i++) {
				if(Stopwords.isStopword(words.get(i)) || isNoiseWord(words.get(i))) {
					words.remove(i);
					i--;
				}
			}
			return words;
		}
		public String getDocName() {
			return docName;
		}
		public int[] getTerms() {
			return terms;
		}
		public boolean isNoiseWord(String string) {
			// TODO Auto-generated method stub
			string = string.toLowerCase().trim();
			Pattern MY_PATTERN = Pattern.compile(".*[a-zA-Z]+.*");
			Matcher m = MY_PATTERN.matcher(string);
			// filter @xxx and URL
			if(string.matches(".*www\\..*") || string.matches(".*\\.com.*") || 
					string.matches(".*http:.*") )
				return true;
			if (!m.matches()) {
				return true;
			} else
				return false;
		}
	}
}
